{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words: 888723\n",
      "Vocabulary size: 9251\n",
      "Word preprocessing completed ...\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:1346: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "Data Shuffled\n",
      "Epoch 1/10 | Batch 0/2067 | train loss: 3.6443\n",
      "Nearest to [six]: dialysis, with, whichever, expressed, exported,\n",
      "Nearest to [gold]: whittle, hollywood, sight, cut, counts,\n",
      "Nearest to [japan]: lipper, bursts, consortium, generous, ruble,\n",
      "Nearest to [college]: silly, operating, tends, evil, regarding,\n",
      "Epoch 1/10 | Batch 50/2067 | train loss: 3.4746\n",
      "Epoch 1/10 | Batch 100/2067 | train loss: 3.5896\n",
      "Epoch 1/10 | Batch 150/2067 | train loss: 3.6596\n",
      "Epoch 1/10 | Batch 200/2067 | train loss: 3.8237\n",
      "Epoch 1/10 | Batch 250/2067 | train loss: 3.2563\n",
      "Epoch 1/10 | Batch 300/2067 | train loss: 3.8517\n",
      "Epoch 1/10 | Batch 350/2067 | train loss: 3.4690\n",
      "Epoch 1/10 | Batch 400/2067 | train loss: 3.6462\n",
      "Epoch 1/10 | Batch 450/2067 | train loss: 3.6095\n",
      "Epoch 1/10 | Batch 500/2067 | train loss: 3.1144\n",
      "Epoch 1/10 | Batch 550/2067 | train loss: 3.0948\n",
      "Epoch 1/10 | Batch 600/2067 | train loss: 4.0774\n",
      "Epoch 1/10 | Batch 650/2067 | train loss: 3.2511\n",
      "Epoch 1/10 | Batch 700/2067 | train loss: 4.1016\n",
      "Epoch 1/10 | Batch 750/2067 | train loss: 3.1094\n",
      "Epoch 1/10 | Batch 800/2067 | train loss: 3.5379\n",
      "Epoch 1/10 | Batch 850/2067 | train loss: 4.1339\n",
      "Epoch 1/10 | Batch 900/2067 | train loss: 3.0653\n",
      "Epoch 1/10 | Batch 950/2067 | train loss: 4.0296\n",
      "Epoch 1/10 | Batch 1000/2067 | train loss: 3.1558\n",
      "Nearest to [six]: with, ceramic, city, appointments, however,\n",
      "Nearest to [gold]: ounce, ounces, moderate, settled, thursday,\n",
      "Nearest to [japan]: midday, finnair, respective, defenses, data,\n",
      "Nearest to [college]: undermine, clearance, afford, sullivan, struggling,\n",
      "Epoch 1/10 | Batch 1050/2067 | train loss: 2.8756\n",
      "Epoch 1/10 | Batch 1100/2067 | train loss: 3.0780\n",
      "Epoch 1/10 | Batch 1150/2067 | train loss: 2.8811\n",
      "Epoch 1/10 | Batch 1200/2067 | train loss: 3.0579\n",
      "Epoch 1/10 | Batch 1250/2067 | train loss: 3.2635\n",
      "Epoch 1/10 | Batch 1300/2067 | train loss: 3.2635\n",
      "Epoch 1/10 | Batch 1350/2067 | train loss: 3.5319\n",
      "Epoch 1/10 | Batch 1400/2067 | train loss: 3.9270\n",
      "Epoch 1/10 | Batch 1450/2067 | train loss: 3.2062\n",
      "Epoch 1/10 | Batch 1500/2067 | train loss: 3.3107\n",
      "Epoch 1/10 | Batch 1550/2067 | train loss: 3.1381\n",
      "Epoch 1/10 | Batch 1600/2067 | train loss: 3.3842\n",
      "Epoch 1/10 | Batch 1650/2067 | train loss: 2.8298\n",
      "Epoch 1/10 | Batch 1700/2067 | train loss: 3.2491\n",
      "Epoch 1/10 | Batch 1750/2067 | train loss: 3.1486\n",
      "Epoch 1/10 | Batch 1800/2067 | train loss: 3.7933\n",
      "Epoch 1/10 | Batch 1850/2067 | train loss: 3.9528\n",
      "Epoch 1/10 | Batch 1900/2067 | train loss: 2.9750\n",
      "Epoch 1/10 | Batch 1950/2067 | train loss: 3.5366\n",
      "Epoch 1/10 | Batch 2000/2067 | train loss: 3.1466\n",
      "Nearest to [six]: ceramic, with, city, disposable, however,\n",
      "Nearest to [gold]: ounce, silver, moderate, settled, december,\n",
      "Nearest to [japan]: data, explosive, examples, bloody, diplomats,\n",
      "Nearest to [college]: teach, competent, friends, graduates, read,\n",
      "Epoch 1/10 | Batch 2050/2067 | train loss: 2.8455\n",
      "Data Shuffled\n",
      "Epoch 2/10 | Batch 0/2067 | train loss: 3.3966\n",
      "Nearest to [six]: ceramic, with, city, however, disposable,\n",
      "Nearest to [gold]: ounce, silver, moderate, platinum, ounces,\n",
      "Nearest to [japan]: data, explosive, examples, diplomats, bloody,\n",
      "Nearest to [college]: sports, education, children, message, clout,\n",
      "Epoch 2/10 | Batch 50/2067 | train loss: 3.1764\n",
      "Epoch 2/10 | Batch 100/2067 | train loss: 3.0522\n",
      "Epoch 2/10 | Batch 150/2067 | train loss: 3.1345\n",
      "Epoch 2/10 | Batch 200/2067 | train loss: 2.7553\n",
      "Epoch 2/10 | Batch 250/2067 | train loss: 3.2567\n",
      "Epoch 2/10 | Batch 300/2067 | train loss: 3.1375\n",
      "Epoch 2/10 | Batch 350/2067 | train loss: 3.3123\n",
      "Epoch 2/10 | Batch 400/2067 | train loss: 3.4592\n",
      "Epoch 2/10 | Batch 450/2067 | train loss: 2.9992\n",
      "Epoch 2/10 | Batch 500/2067 | train loss: 3.1605\n",
      "Epoch 2/10 | Batch 550/2067 | train loss: 3.2295\n",
      "Epoch 2/10 | Batch 600/2067 | train loss: 2.8412\n",
      "Epoch 2/10 | Batch 650/2067 | train loss: 2.8604\n",
      "Epoch 2/10 | Batch 700/2067 | train loss: 2.8625\n",
      "Epoch 2/10 | Batch 750/2067 | train loss: 3.5012\n",
      "Epoch 2/10 | Batch 800/2067 | train loss: 2.7708\n",
      "Epoch 2/10 | Batch 850/2067 | train loss: 2.5662\n",
      "Epoch 2/10 | Batch 900/2067 | train loss: 2.8089\n",
      "Epoch 2/10 | Batch 950/2067 | train loss: 3.0256\n",
      "Epoch 2/10 | Batch 1000/2067 | train loss: 3.7154\n",
      "Nearest to [six]: with, city, however, phones, propelled,\n",
      "Nearest to [gold]: ounce, silver, ounces, moderate, platinum,\n",
      "Nearest to [japan]: explosive, data, bloody, added, stolen,\n",
      "Nearest to [college]: children, education, parents, tells, basketball,\n",
      "Epoch 2/10 | Batch 1050/2067 | train loss: 2.7271\n",
      "Epoch 2/10 | Batch 1100/2067 | train loss: 2.9770\n",
      "Epoch 2/10 | Batch 1150/2067 | train loss: 2.7749\n",
      "Epoch 2/10 | Batch 1200/2067 | train loss: 3.2475\n",
      "Epoch 2/10 | Batch 1250/2067 | train loss: 3.0488\n",
      "Epoch 2/10 | Batch 1300/2067 | train loss: 3.0499\n",
      "Epoch 2/10 | Batch 1350/2067 | train loss: 3.3071\n",
      "Epoch 2/10 | Batch 1400/2067 | train loss: 3.3878\n",
      "Epoch 2/10 | Batch 1450/2067 | train loss: 2.7863\n",
      "Epoch 2/10 | Batch 1500/2067 | train loss: 2.3605\n",
      "Epoch 2/10 | Batch 1550/2067 | train loss: 2.6918\n",
      "Epoch 2/10 | Batch 1600/2067 | train loss: 3.0951\n",
      "Epoch 2/10 | Batch 1650/2067 | train loss: 2.9120\n",
      "Epoch 2/10 | Batch 1700/2067 | train loss: 2.9951\n",
      "Epoch 2/10 | Batch 1750/2067 | train loss: 2.7030\n",
      "Epoch 2/10 | Batch 1800/2067 | train loss: 2.9307\n",
      "Epoch 2/10 | Batch 1850/2067 | train loss: 2.9520\n",
      "Epoch 2/10 | Batch 1900/2067 | train loss: 2.8993\n",
      "Epoch 2/10 | Batch 1950/2067 | train loss: 2.7875\n",
      "Epoch 2/10 | Batch 2000/2067 | train loss: 3.5236\n",
      "Nearest to [six]: with, city, however, focused, phones,\n",
      "Nearest to [gold]: ounce, silver, ounces, platinum, december,\n",
      "Nearest to [japan]: respectable, data, examples, explosive, edt,\n",
      "Nearest to [college]: parents, children, education, basketball, child,\n",
      "Epoch 2/10 | Batch 2050/2067 | train loss: 2.4500\n",
      "Data Shuffled\n",
      "Epoch 3/10 | Batch 0/2067 | train loss: 3.0410\n",
      "Nearest to [six]: with, city, however, focused, devaluation,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, december,\n",
      "Nearest to [japan]: respectable, data, examples, explosive, edt,\n",
      "Nearest to [college]: parents, children, child, education, journalism,\n",
      "Epoch 3/10 | Batch 50/2067 | train loss: 2.9259\n",
      "Epoch 3/10 | Batch 100/2067 | train loss: 2.2054\n",
      "Epoch 3/10 | Batch 150/2067 | train loss: 2.9451\n",
      "Epoch 3/10 | Batch 200/2067 | train loss: 2.9528\n",
      "Epoch 3/10 | Batch 250/2067 | train loss: 2.9599\n",
      "Epoch 3/10 | Batch 300/2067 | train loss: 2.6125\n",
      "Epoch 3/10 | Batch 350/2067 | train loss: 2.6345\n",
      "Epoch 3/10 | Batch 400/2067 | train loss: 2.2841\n",
      "Epoch 3/10 | Batch 450/2067 | train loss: 2.3963\n",
      "Epoch 3/10 | Batch 500/2067 | train loss: 3.2233\n",
      "Epoch 3/10 | Batch 550/2067 | train loss: 2.2181\n",
      "Epoch 3/10 | Batch 600/2067 | train loss: 2.4862\n",
      "Epoch 3/10 | Batch 650/2067 | train loss: 2.8347\n",
      "Epoch 3/10 | Batch 700/2067 | train loss: 2.7404\n",
      "Epoch 3/10 | Batch 750/2067 | train loss: 3.2410\n",
      "Epoch 3/10 | Batch 800/2067 | train loss: 2.5056\n",
      "Epoch 3/10 | Batch 850/2067 | train loss: 2.1233\n",
      "Epoch 3/10 | Batch 900/2067 | train loss: 2.4502\n",
      "Epoch 3/10 | Batch 950/2067 | train loss: 2.3792\n",
      "Epoch 3/10 | Batch 1000/2067 | train loss: 2.7508\n",
      "Nearest to [six]: with, city, focused, however, alongside,\n",
      "Nearest to [gold]: ounce, silver, ounces, platinum, december,\n",
      "Nearest to [japan]: examples, data, respectable, added, craze,\n",
      "Nearest to [college]: parents, basketball, child, football, children,\n",
      "Epoch 3/10 | Batch 1050/2067 | train loss: 2.3061\n",
      "Epoch 3/10 | Batch 1100/2067 | train loss: 2.9616\n",
      "Epoch 3/10 | Batch 1150/2067 | train loss: 2.5332\n",
      "Epoch 3/10 | Batch 1200/2067 | train loss: 2.4147\n",
      "Epoch 3/10 | Batch 1250/2067 | train loss: 2.5262\n",
      "Epoch 3/10 | Batch 1300/2067 | train loss: 2.6555\n",
      "Epoch 3/10 | Batch 1350/2067 | train loss: 2.8747\n",
      "Epoch 3/10 | Batch 1400/2067 | train loss: 2.6218\n",
      "Epoch 3/10 | Batch 1450/2067 | train loss: 2.7461\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/10 | Batch 1500/2067 | train loss: 2.9629\n",
      "Epoch 3/10 | Batch 1550/2067 | train loss: 2.1935\n",
      "Epoch 3/10 | Batch 1600/2067 | train loss: 2.2869\n",
      "Epoch 3/10 | Batch 1650/2067 | train loss: 2.4692\n",
      "Epoch 3/10 | Batch 1700/2067 | train loss: 2.3022\n",
      "Epoch 3/10 | Batch 1750/2067 | train loss: 2.0526\n",
      "Epoch 3/10 | Batch 1800/2067 | train loss: 2.5889\n",
      "Epoch 3/10 | Batch 1850/2067 | train loss: 2.8986\n",
      "Epoch 3/10 | Batch 1900/2067 | train loss: 2.2747\n",
      "Epoch 3/10 | Batch 1950/2067 | train loss: 2.8979\n",
      "Epoch 3/10 | Batch 2000/2067 | train loss: 2.5023\n",
      "Nearest to [six]: with, city, however, break, alongside,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, examples, data, revamped, added,\n",
      "Nearest to [college]: parents, basketball, child, football, children,\n",
      "Epoch 3/10 | Batch 2050/2067 | train loss: 2.4079\n",
      "Data Shuffled\n",
      "Epoch 4/10 | Batch 0/2067 | train loss: 1.9417\n",
      "Nearest to [six]: with, city, however, break, alongside,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, december,\n",
      "Nearest to [japan]: respectable, examples, data, added, revamped,\n",
      "Nearest to [college]: parents, basketball, child, football, children,\n",
      "Epoch 4/10 | Batch 50/2067 | train loss: 2.1167\n",
      "Epoch 4/10 | Batch 100/2067 | train loss: 2.6247\n",
      "Epoch 4/10 | Batch 150/2067 | train loss: 1.8751\n",
      "Epoch 4/10 | Batch 200/2067 | train loss: 2.4462\n",
      "Epoch 4/10 | Batch 250/2067 | train loss: 2.0544\n",
      "Epoch 4/10 | Batch 300/2067 | train loss: 2.2055\n",
      "Epoch 4/10 | Batch 350/2067 | train loss: 2.1823\n",
      "Epoch 4/10 | Batch 400/2067 | train loss: 2.9062\n",
      "Epoch 4/10 | Batch 450/2067 | train loss: 2.1563\n",
      "Epoch 4/10 | Batch 500/2067 | train loss: 2.1989\n",
      "Epoch 4/10 | Batch 550/2067 | train loss: 2.5016\n",
      "Epoch 4/10 | Batch 600/2067 | train loss: 2.7076\n",
      "Epoch 4/10 | Batch 650/2067 | train loss: 1.9582\n",
      "Epoch 4/10 | Batch 700/2067 | train loss: 2.4557\n",
      "Epoch 4/10 | Batch 750/2067 | train loss: 2.5608\n",
      "Epoch 4/10 | Batch 800/2067 | train loss: 2.2844\n",
      "Epoch 4/10 | Batch 850/2067 | train loss: 2.8526\n",
      "Epoch 4/10 | Batch 900/2067 | train loss: 2.0631\n",
      "Epoch 4/10 | Batch 950/2067 | train loss: 2.5353\n",
      "Epoch 4/10 | Batch 1000/2067 | train loss: 2.3182\n",
      "Nearest to [six]: with, city, focused, however, alongside,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, examples, data, added, modify,\n",
      "Nearest to [college]: parents, child, football, basketball, sports,\n",
      "Epoch 4/10 | Batch 1050/2067 | train loss: 2.3654\n",
      "Epoch 4/10 | Batch 1100/2067 | train loss: 2.5246\n",
      "Epoch 4/10 | Batch 1150/2067 | train loss: 2.2296\n",
      "Epoch 4/10 | Batch 1200/2067 | train loss: 2.0883\n",
      "Epoch 4/10 | Batch 1250/2067 | train loss: 1.8624\n",
      "Epoch 4/10 | Batch 1300/2067 | train loss: 1.8118\n",
      "Epoch 4/10 | Batch 1350/2067 | train loss: 2.3700\n",
      "Epoch 4/10 | Batch 1400/2067 | train loss: 2.0617\n",
      "Epoch 4/10 | Batch 1450/2067 | train loss: 2.2547\n",
      "Epoch 4/10 | Batch 1500/2067 | train loss: 2.1189\n",
      "Epoch 4/10 | Batch 1550/2067 | train loss: 1.8777\n",
      "Epoch 4/10 | Batch 1600/2067 | train loss: 1.9629\n",
      "Epoch 4/10 | Batch 1650/2067 | train loss: 2.3583\n",
      "Epoch 4/10 | Batch 1700/2067 | train loss: 2.6733\n",
      "Epoch 4/10 | Batch 1750/2067 | train loss: 2.4592\n",
      "Epoch 4/10 | Batch 1800/2067 | train loss: 2.2072\n",
      "Epoch 4/10 | Batch 1850/2067 | train loss: 2.0386\n",
      "Epoch 4/10 | Batch 1900/2067 | train loss: 1.7273\n",
      "Epoch 4/10 | Batch 1950/2067 | train loss: 2.2820\n",
      "Epoch 4/10 | Batch 2000/2067 | train loss: 2.1770\n",
      "Nearest to [six]: with, city, break, however, wore,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, examples, added, camp,\n",
      "Nearest to [college]: parents, basketball, child, educational, football,\n",
      "Epoch 4/10 | Batch 2050/2067 | train loss: 2.0931\n",
      "Data Shuffled\n",
      "Epoch 5/10 | Batch 0/2067 | train loss: 2.0453\n",
      "Nearest to [six]: with, city, wore, break, however,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, examples, added, craze,\n",
      "Nearest to [college]: basketball, parents, child, educational, football,\n",
      "Epoch 5/10 | Batch 50/2067 | train loss: 2.3510\n",
      "Epoch 5/10 | Batch 100/2067 | train loss: 2.0600\n",
      "Epoch 5/10 | Batch 150/2067 | train loss: 2.2108\n",
      "Epoch 5/10 | Batch 200/2067 | train loss: 2.1164\n",
      "Epoch 5/10 | Batch 250/2067 | train loss: 2.1789\n",
      "Epoch 5/10 | Batch 300/2067 | train loss: 1.7271\n",
      "Epoch 5/10 | Batch 350/2067 | train loss: 2.2819\n",
      "Epoch 5/10 | Batch 400/2067 | train loss: 2.0343\n",
      "Epoch 5/10 | Batch 450/2067 | train loss: 2.1125\n",
      "Epoch 5/10 | Batch 500/2067 | train loss: 2.0817\n",
      "Epoch 5/10 | Batch 550/2067 | train loss: 2.2394\n",
      "Epoch 5/10 | Batch 600/2067 | train loss: 2.3568\n",
      "Epoch 5/10 | Batch 650/2067 | train loss: 1.9680\n",
      "Epoch 5/10 | Batch 700/2067 | train loss: 1.8118\n",
      "Epoch 5/10 | Batch 750/2067 | train loss: 2.1874\n",
      "Epoch 5/10 | Batch 800/2067 | train loss: 1.7937\n",
      "Epoch 5/10 | Batch 850/2067 | train loss: 1.4916\n",
      "Epoch 5/10 | Batch 900/2067 | train loss: 2.1609\n",
      "Epoch 5/10 | Batch 950/2067 | train loss: 2.1721\n",
      "Epoch 5/10 | Batch 1000/2067 | train loss: 2.2448\n",
      "Nearest to [six]: with, city, frame, alongside, however,\n",
      "Nearest to [gold]: ounce, silver, ounces, platinum, bullion,\n",
      "Nearest to [japan]: respectable, data, added, modify, examples,\n",
      "Nearest to [college]: basketball, parents, child, football, sports,\n",
      "Epoch 5/10 | Batch 1050/2067 | train loss: 1.8277\n",
      "Epoch 5/10 | Batch 1100/2067 | train loss: 2.4328\n",
      "Epoch 5/10 | Batch 1150/2067 | train loss: 1.8187\n",
      "Epoch 5/10 | Batch 1200/2067 | train loss: 2.2354\n",
      "Epoch 5/10 | Batch 1250/2067 | train loss: 1.9890\n",
      "Epoch 5/10 | Batch 1300/2067 | train loss: 2.1727\n",
      "Epoch 5/10 | Batch 1350/2067 | train loss: 1.8684\n",
      "Epoch 5/10 | Batch 1400/2067 | train loss: 2.1469\n",
      "Epoch 5/10 | Batch 1450/2067 | train loss: 1.8052\n",
      "Epoch 5/10 | Batch 1500/2067 | train loss: 1.9575\n",
      "Epoch 5/10 | Batch 1550/2067 | train loss: 2.2353\n",
      "Epoch 5/10 | Batch 1600/2067 | train loss: 2.2837\n",
      "Epoch 5/10 | Batch 1650/2067 | train loss: 2.3095\n",
      "Epoch 5/10 | Batch 1700/2067 | train loss: 2.4189\n",
      "Epoch 5/10 | Batch 1750/2067 | train loss: 1.9035\n",
      "Epoch 5/10 | Batch 1800/2067 | train loss: 2.0406\n",
      "Epoch 5/10 | Batch 1850/2067 | train loss: 2.0068\n",
      "Epoch 5/10 | Batch 1900/2067 | train loss: 2.0678\n",
      "Epoch 5/10 | Batch 1950/2067 | train loss: 2.1948\n",
      "Epoch 5/10 | Batch 2000/2067 | train loss: 1.7741\n",
      "Nearest to [six]: with, frame, city, alongside, however,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, factor, examples, added,\n",
      "Nearest to [college]: basketball, parents, child, sports, football,\n",
      "Epoch 5/10 | Batch 2050/2067 | train loss: 2.0537\n",
      "Data Shuffled\n",
      "Epoch 6/10 | Batch 0/2067 | train loss: 1.5946\n",
      "Nearest to [six]: with, frame, city, wore, however,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, examples, factor, added,\n",
      "Nearest to [college]: basketball, parents, child, sports, football,\n",
      "Epoch 6/10 | Batch 50/2067 | train loss: 2.2006\n",
      "Epoch 6/10 | Batch 100/2067 | train loss: 2.1431\n",
      "Epoch 6/10 | Batch 150/2067 | train loss: 1.6513\n",
      "Epoch 6/10 | Batch 200/2067 | train loss: 1.4906\n",
      "Epoch 6/10 | Batch 250/2067 | train loss: 1.6957\n",
      "Epoch 6/10 | Batch 300/2067 | train loss: 1.8571\n",
      "Epoch 6/10 | Batch 350/2067 | train loss: 1.9835\n",
      "Epoch 6/10 | Batch 400/2067 | train loss: 1.4948\n",
      "Epoch 6/10 | Batch 450/2067 | train loss: 1.7511\n",
      "Epoch 6/10 | Batch 500/2067 | train loss: 1.9936\n",
      "Epoch 6/10 | Batch 550/2067 | train loss: 1.9909\n",
      "Epoch 6/10 | Batch 600/2067 | train loss: 1.9293\n",
      "Epoch 6/10 | Batch 650/2067 | train loss: 1.9334\n",
      "Epoch 6/10 | Batch 700/2067 | train loss: 1.4065\n",
      "Epoch 6/10 | Batch 750/2067 | train loss: 1.3690\n",
      "Epoch 6/10 | Batch 800/2067 | train loss: 1.8454\n",
      "Epoch 6/10 | Batch 850/2067 | train loss: 1.5891\n",
      "Epoch 6/10 | Batch 900/2067 | train loss: 1.8819\n",
      "Epoch 6/10 | Batch 950/2067 | train loss: 1.7865\n",
      "Epoch 6/10 | Batch 1000/2067 | train loss: 1.9304\n",
      "Nearest to [six]: with, frame, city, wore, focused,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, examples, added, factor,\n",
      "Nearest to [college]: basketball, parents, child, football, sports,\n",
      "Epoch 6/10 | Batch 1050/2067 | train loss: 2.3954\n",
      "Epoch 6/10 | Batch 1100/2067 | train loss: 1.9294\n",
      "Epoch 6/10 | Batch 1150/2067 | train loss: 1.7686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/10 | Batch 1200/2067 | train loss: 1.7040\n",
      "Epoch 6/10 | Batch 1250/2067 | train loss: 1.6646\n",
      "Epoch 6/10 | Batch 1300/2067 | train loss: 2.1861\n",
      "Epoch 6/10 | Batch 1350/2067 | train loss: 2.0562\n",
      "Epoch 6/10 | Batch 1400/2067 | train loss: 1.4214\n",
      "Epoch 6/10 | Batch 1450/2067 | train loss: 2.0239\n",
      "Epoch 6/10 | Batch 1500/2067 | train loss: 2.0954\n",
      "Epoch 6/10 | Batch 1550/2067 | train loss: 1.5561\n",
      "Epoch 6/10 | Batch 1600/2067 | train loss: 1.7400\n",
      "Epoch 6/10 | Batch 1650/2067 | train loss: 1.7912\n",
      "Epoch 6/10 | Batch 1700/2067 | train loss: 2.0884\n",
      "Epoch 6/10 | Batch 1750/2067 | train loss: 2.5915\n",
      "Epoch 6/10 | Batch 1800/2067 | train loss: 1.7102\n",
      "Epoch 6/10 | Batch 1850/2067 | train loss: 1.7821\n",
      "Epoch 6/10 | Batch 1900/2067 | train loss: 1.5400\n",
      "Epoch 6/10 | Batch 1950/2067 | train loss: 1.8450\n",
      "Epoch 6/10 | Batch 2000/2067 | train loss: 1.9621\n",
      "Nearest to [six]: frame, with, wore, city, focused,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, factor, alternative, added,\n",
      "Nearest to [college]: basketball, football, parents, sports, teaches,\n",
      "Epoch 6/10 | Batch 2050/2067 | train loss: 1.7423\n",
      "Data Shuffled\n",
      "Epoch 7/10 | Batch 0/2067 | train loss: 1.8702\n",
      "Nearest to [six]: frame, with, wore, city, focused,\n",
      "Nearest to [gold]: ounce, silver, platinum, ounces, bullion,\n",
      "Nearest to [japan]: respectable, data, factor, alternative, added,\n",
      "Nearest to [college]: basketball, parents, football, sports, child,\n",
      "Epoch 7/10 | Batch 50/2067 | train loss: 1.5049\n",
      "Epoch 7/10 | Batch 100/2067 | train loss: 1.8000\n",
      "Epoch 7/10 | Batch 150/2067 | train loss: 1.2394\n",
      "Epoch 7/10 | Batch 200/2067 | train loss: 1.6441\n",
      "Epoch 7/10 | Batch 250/2067 | train loss: 1.7719\n",
      "Epoch 7/10 | Batch 300/2067 | train loss: 1.5165\n",
      "Epoch 7/10 | Batch 350/2067 | train loss: 1.3886\n",
      "Epoch 7/10 | Batch 400/2067 | train loss: 1.5705\n",
      "Epoch 7/10 | Batch 450/2067 | train loss: 1.3738\n",
      "Epoch 7/10 | Batch 500/2067 | train loss: 1.4512\n",
      "Epoch 7/10 | Batch 550/2067 | train loss: 1.5612\n",
      "Epoch 7/10 | Batch 600/2067 | train loss: 1.8452\n",
      "Epoch 7/10 | Batch 650/2067 | train loss: 1.6073\n",
      "Epoch 7/10 | Batch 700/2067 | train loss: 1.9472\n",
      "Epoch 7/10 | Batch 750/2067 | train loss: 1.5799\n",
      "Epoch 7/10 | Batch 800/2067 | train loss: 2.1880\n",
      "Epoch 7/10 | Batch 850/2067 | train loss: 1.6284\n",
      "Epoch 7/10 | Batch 900/2067 | train loss: 1.5742\n",
      "Epoch 7/10 | Batch 950/2067 | train loss: 1.9144\n",
      "Epoch 7/10 | Batch 1000/2067 | train loss: 1.3977\n",
      "Nearest to [six]: frame, with, focused, city, guzman,\n",
      "Nearest to [gold]: ounce, silver, platinum, bullion, ounces,\n",
      "Nearest to [japan]: respectable, factor, data, harsh, added,\n",
      "Nearest to [college]: basketball, sports, parents, football, child,\n",
      "Epoch 7/10 | Batch 1050/2067 | train loss: 1.4720\n",
      "Epoch 7/10 | Batch 1100/2067 | train loss: 1.7534\n",
      "Epoch 7/10 | Batch 1150/2067 | train loss: 1.3053\n",
      "Epoch 7/10 | Batch 1200/2067 | train loss: 1.6451\n",
      "Epoch 7/10 | Batch 1250/2067 | train loss: 1.2607\n",
      "Epoch 7/10 | Batch 1300/2067 | train loss: 1.7652\n",
      "Epoch 7/10 | Batch 1350/2067 | train loss: 1.4001\n",
      "Epoch 7/10 | Batch 1400/2067 | train loss: 1.4096\n",
      "Epoch 7/10 | Batch 1450/2067 | train loss: 1.9612\n",
      "Epoch 7/10 | Batch 1500/2067 | train loss: 1.4432\n",
      "Epoch 7/10 | Batch 1550/2067 | train loss: 1.6335\n",
      "Epoch 7/10 | Batch 1600/2067 | train loss: 1.3403\n",
      "Epoch 7/10 | Batch 1650/2067 | train loss: 1.7977\n",
      "Epoch 7/10 | Batch 1700/2067 | train loss: 1.7694\n",
      "Epoch 7/10 | Batch 1750/2067 | train loss: 1.6220\n",
      "Epoch 7/10 | Batch 1800/2067 | train loss: 1.3159\n",
      "Epoch 7/10 | Batch 1850/2067 | train loss: 1.3039\n",
      "Epoch 7/10 | Batch 1900/2067 | train loss: 1.5266\n",
      "Epoch 7/10 | Batch 1950/2067 | train loss: 1.8391\n",
      "Epoch 7/10 | Batch 2000/2067 | train loss: 2.2417\n",
      "Nearest to [six]: frame, with, guzman, city, wore,\n",
      "Nearest to [gold]: ounce, silver, platinum, bullion, ounces,\n",
      "Nearest to [japan]: respectable, data, factor, alternative, added,\n",
      "Nearest to [college]: basketball, football, parents, teaches, sports,\n",
      "Epoch 7/10 | Batch 2050/2067 | train loss: 1.5390\n",
      "Data Shuffled\n",
      "Epoch 8/10 | Batch 0/2067 | train loss: 1.9312\n",
      "Nearest to [six]: frame, with, guzman, city, focused,\n",
      "Nearest to [gold]: ounce, silver, platinum, bullion, ounces,\n",
      "Nearest to [japan]: respectable, data, factor, alternative, added,\n",
      "Nearest to [college]: basketball, football, parents, teaches, sports,\n",
      "Epoch 8/10 | Batch 50/2067 | train loss: 1.4714\n",
      "Epoch 8/10 | Batch 100/2067 | train loss: 1.3124\n",
      "Epoch 8/10 | Batch 150/2067 | train loss: 1.1350\n",
      "Epoch 8/10 | Batch 200/2067 | train loss: 1.5859\n",
      "Epoch 8/10 | Batch 250/2067 | train loss: 1.2796\n",
      "Epoch 8/10 | Batch 300/2067 | train loss: 1.1917\n",
      "Epoch 8/10 | Batch 350/2067 | train loss: 1.4378\n",
      "Epoch 8/10 | Batch 400/2067 | train loss: 1.7847\n",
      "Epoch 8/10 | Batch 450/2067 | train loss: 1.3350\n",
      "Epoch 8/10 | Batch 500/2067 | train loss: 1.1260\n",
      "Epoch 8/10 | Batch 550/2067 | train loss: 1.2366\n",
      "Epoch 8/10 | Batch 600/2067 | train loss: 1.4710\n",
      "Epoch 8/10 | Batch 650/2067 | train loss: 1.3968\n",
      "Epoch 8/10 | Batch 700/2067 | train loss: 1.3501\n",
      "Epoch 8/10 | Batch 750/2067 | train loss: 1.4177\n",
      "Epoch 8/10 | Batch 800/2067 | train loss: 1.1315\n",
      "Epoch 8/10 | Batch 850/2067 | train loss: 1.4130\n",
      "Epoch 8/10 | Batch 900/2067 | train loss: 1.5017\n",
      "Epoch 8/10 | Batch 950/2067 | train loss: 1.9594\n",
      "Epoch 8/10 | Batch 1000/2067 | train loss: 1.2606\n",
      "Nearest to [six]: frame, with, guzman, city, focused,\n",
      "Nearest to [gold]: ounce, silver, platinum, bullion, ounces,\n",
      "Nearest to [japan]: respectable, data, factor, alternative, added,\n",
      "Nearest to [college]: basketball, football, parents, sports, teaches,\n",
      "Epoch 8/10 | Batch 1050/2067 | train loss: 1.1988\n",
      "Epoch 8/10 | Batch 1100/2067 | train loss: 1.1629\n",
      "Epoch 8/10 | Batch 1150/2067 | train loss: 1.6745\n",
      "Epoch 8/10 | Batch 1200/2067 | train loss: 1.5801\n",
      "Epoch 8/10 | Batch 1250/2067 | train loss: 1.5808\n",
      "Epoch 8/10 | Batch 1300/2067 | train loss: 1.3521\n",
      "Epoch 8/10 | Batch 1350/2067 | train loss: 1.7741\n",
      "Epoch 8/10 | Batch 1400/2067 | train loss: 1.4310\n",
      "Epoch 8/10 | Batch 1450/2067 | train loss: 1.4572\n",
      "Epoch 8/10 | Batch 1500/2067 | train loss: 1.3413\n",
      "Epoch 8/10 | Batch 1550/2067 | train loss: 1.5712\n",
      "Epoch 8/10 | Batch 1600/2067 | train loss: 1.8063\n",
      "Epoch 8/10 | Batch 1650/2067 | train loss: 1.5775\n",
      "Epoch 8/10 | Batch 1700/2067 | train loss: 0.9803\n",
      "Epoch 8/10 | Batch 1750/2067 | train loss: 1.4585\n",
      "Epoch 8/10 | Batch 1800/2067 | train loss: 1.5152\n",
      "Epoch 8/10 | Batch 1850/2067 | train loss: 1.2386\n",
      "Epoch 8/10 | Batch 1900/2067 | train loss: 1.4939\n",
      "Epoch 8/10 | Batch 1950/2067 | train loss: 1.0489\n",
      "Epoch 8/10 | Batch 2000/2067 | train loss: 1.3704\n",
      "Nearest to [six]: frame, with, guzman, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: respectable, factor, data, harsh, original,\n",
      "Nearest to [college]: basketball, football, voter, parents, sports,\n",
      "Epoch 8/10 | Batch 2050/2067 | train loss: 1.5829\n",
      "Data Shuffled\n",
      "Epoch 9/10 | Batch 0/2067 | train loss: 1.3908\n",
      "Nearest to [six]: frame, with, guzman, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: respectable, factor, data, harsh, alternative,\n",
      "Nearest to [college]: basketball, football, voter, parents, sports,\n",
      "Epoch 9/10 | Batch 50/2067 | train loss: 1.6979\n",
      "Epoch 9/10 | Batch 100/2067 | train loss: 1.4778\n",
      "Epoch 9/10 | Batch 150/2067 | train loss: 1.1252\n",
      "Epoch 9/10 | Batch 200/2067 | train loss: 0.9863\n",
      "Epoch 9/10 | Batch 250/2067 | train loss: 1.1324\n",
      "Epoch 9/10 | Batch 300/2067 | train loss: 1.4998\n",
      "Epoch 9/10 | Batch 350/2067 | train loss: 1.7091\n",
      "Epoch 9/10 | Batch 400/2067 | train loss: 1.2321\n",
      "Epoch 9/10 | Batch 450/2067 | train loss: 1.4808\n",
      "Epoch 9/10 | Batch 500/2067 | train loss: 1.1991\n",
      "Epoch 9/10 | Batch 550/2067 | train loss: 0.8404\n",
      "Epoch 9/10 | Batch 600/2067 | train loss: 1.0459\n",
      "Epoch 9/10 | Batch 650/2067 | train loss: 1.4680\n",
      "Epoch 9/10 | Batch 700/2067 | train loss: 1.5746\n",
      "Epoch 9/10 | Batch 750/2067 | train loss: 1.0610\n",
      "Epoch 9/10 | Batch 800/2067 | train loss: 1.3790\n",
      "Epoch 9/10 | Batch 850/2067 | train loss: 1.4131\n",
      "Epoch 9/10 | Batch 900/2067 | train loss: 1.2583\n",
      "Epoch 9/10 | Batch 950/2067 | train loss: 1.2361\n",
      "Epoch 9/10 | Batch 1000/2067 | train loss: 0.9947\n",
      "Nearest to [six]: frame, with, guzman, focused, city,\n",
      "Nearest to [gold]: ounce, silver, platinum, bullion, ounces,\n",
      "Nearest to [japan]: factor, respectable, data, original, harsh,\n",
      "Nearest to [college]: basketball, football, voter, sports, parents,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/10 | Batch 1050/2067 | train loss: 1.4405\n",
      "Epoch 9/10 | Batch 1100/2067 | train loss: 1.0601\n",
      "Epoch 9/10 | Batch 1150/2067 | train loss: 1.2051\n",
      "Epoch 9/10 | Batch 1200/2067 | train loss: 0.9359\n",
      "Epoch 9/10 | Batch 1250/2067 | train loss: 1.0791\n",
      "Epoch 9/10 | Batch 1300/2067 | train loss: 1.1137\n",
      "Epoch 9/10 | Batch 1350/2067 | train loss: 0.9766\n",
      "Epoch 9/10 | Batch 1400/2067 | train loss: 1.3143\n",
      "Epoch 9/10 | Batch 1450/2067 | train loss: 1.1378\n",
      "Epoch 9/10 | Batch 1500/2067 | train loss: 1.0154\n",
      "Epoch 9/10 | Batch 1550/2067 | train loss: 1.1831\n",
      "Epoch 9/10 | Batch 1600/2067 | train loss: 1.1688\n",
      "Epoch 9/10 | Batch 1650/2067 | train loss: 1.2535\n",
      "Epoch 9/10 | Batch 1700/2067 | train loss: 1.3225\n",
      "Epoch 9/10 | Batch 1750/2067 | train loss: 1.0476\n",
      "Epoch 9/10 | Batch 1800/2067 | train loss: 1.0396\n",
      "Epoch 9/10 | Batch 1850/2067 | train loss: 1.2481\n",
      "Epoch 9/10 | Batch 1900/2067 | train loss: 1.0815\n",
      "Epoch 9/10 | Batch 1950/2067 | train loss: 0.9650\n",
      "Epoch 9/10 | Batch 2000/2067 | train loss: 1.4820\n",
      "Nearest to [six]: frame, with, guzman, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: factor, respectable, data, alternative, harsh,\n",
      "Nearest to [college]: basketball, football, voter, sports, education,\n",
      "Epoch 9/10 | Batch 2050/2067 | train loss: 1.1713\n",
      "Data Shuffled\n",
      "Epoch 10/10 | Batch 0/2067 | train loss: 0.8869\n",
      "Nearest to [six]: frame, with, guzman, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: factor, respectable, data, alternative, original,\n",
      "Nearest to [college]: basketball, football, voter, sports, parents,\n",
      "Epoch 10/10 | Batch 50/2067 | train loss: 0.9282\n",
      "Epoch 10/10 | Batch 100/2067 | train loss: 0.9005\n",
      "Epoch 10/10 | Batch 150/2067 | train loss: 0.8062\n",
      "Epoch 10/10 | Batch 200/2067 | train loss: 0.7283\n",
      "Epoch 10/10 | Batch 250/2067 | train loss: 0.8108\n",
      "Epoch 10/10 | Batch 300/2067 | train loss: 0.8789\n",
      "Epoch 10/10 | Batch 350/2067 | train loss: 1.2153\n",
      "Epoch 10/10 | Batch 400/2067 | train loss: 0.8302\n",
      "Epoch 10/10 | Batch 450/2067 | train loss: 1.3262\n",
      "Epoch 10/10 | Batch 500/2067 | train loss: 1.0677\n",
      "Epoch 10/10 | Batch 550/2067 | train loss: 1.1854\n",
      "Epoch 10/10 | Batch 600/2067 | train loss: 0.8950\n",
      "Epoch 10/10 | Batch 650/2067 | train loss: 0.8598\n",
      "Epoch 10/10 | Batch 700/2067 | train loss: 1.1922\n",
      "Epoch 10/10 | Batch 750/2067 | train loss: 0.8279\n",
      "Epoch 10/10 | Batch 800/2067 | train loss: 0.8874\n",
      "Epoch 10/10 | Batch 850/2067 | train loss: 1.2355\n",
      "Epoch 10/10 | Batch 900/2067 | train loss: 1.1783\n",
      "Epoch 10/10 | Batch 950/2067 | train loss: 1.1313\n",
      "Epoch 10/10 | Batch 1000/2067 | train loss: 1.1302\n",
      "Nearest to [six]: frame, guzman, with, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: factor, respectable, data, completely, alternative,\n",
      "Nearest to [college]: basketball, football, voter, sports, parents,\n",
      "Epoch 10/10 | Batch 1050/2067 | train loss: 1.0273\n",
      "Epoch 10/10 | Batch 1100/2067 | train loss: 1.1094\n",
      "Epoch 10/10 | Batch 1150/2067 | train loss: 1.1996\n",
      "Epoch 10/10 | Batch 1200/2067 | train loss: 0.9718\n",
      "Epoch 10/10 | Batch 1250/2067 | train loss: 0.6418\n",
      "Epoch 10/10 | Batch 1300/2067 | train loss: 1.0092\n",
      "Epoch 10/10 | Batch 1350/2067 | train loss: 0.6637\n",
      "Epoch 10/10 | Batch 1400/2067 | train loss: 0.8292\n",
      "Epoch 10/10 | Batch 1450/2067 | train loss: 1.0730\n",
      "Epoch 10/10 | Batch 1500/2067 | train loss: 1.0582\n",
      "Epoch 10/10 | Batch 1550/2067 | train loss: 0.9977\n",
      "Epoch 10/10 | Batch 1600/2067 | train loss: 0.7558\n",
      "Epoch 10/10 | Batch 1650/2067 | train loss: 0.8396\n",
      "Epoch 10/10 | Batch 1700/2067 | train loss: 1.0799\n",
      "Epoch 10/10 | Batch 1750/2067 | train loss: 1.2160\n",
      "Epoch 10/10 | Batch 1800/2067 | train loss: 0.8281\n",
      "Epoch 10/10 | Batch 1850/2067 | train loss: 0.8345\n",
      "Epoch 10/10 | Batch 1900/2067 | train loss: 1.0209\n",
      "Epoch 10/10 | Batch 1950/2067 | train loss: 1.1096\n",
      "Epoch 10/10 | Batch 2000/2067 | train loss: 1.2597\n",
      "Nearest to [six]: frame, guzman, with, focused, city,\n",
      "Nearest to [gold]: ounce, silver, bullion, platinum, ounces,\n",
      "Nearest to [japan]: factor, data, respectable, alternative, harsh,\n",
      "Nearest to [college]: basketball, football, voter, sports, parents,\n",
      "Epoch 10/10 | Batch 2050/2067 | train loss: 0.7937\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "from word2vec_cbow import CBOW\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    with open('temp/ptb_train.txt') as f:\n",
    "        text = f.read()\n",
    "    sample_words = ['six', 'gold', 'japan', 'college']\n",
    "\n",
    "    model = CBOW(text, sample_words, useless_words=string.punctuation)\n",
    "    model.fit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
